#!/bin/sh
# -*- tcl -*-
# The next line is executed by /bin/sh, but not tcl \
exec tclsh "$0" ${1+"$@"}

# Author: jiyin@redhat.com
# This is a test run monitor.
# tdom usage: http://wiki.tcl.tk/8984

lappend ::auto_path $::env(HOME)/lib /usr/local/lib /usr/lib64 /usr/lib
package require yaml
package require tdom
package require sqlite3
package require runtestlib 1.1
package require getOpt 3.0
namespace import ::getOpt::* ::runtestlib::*

exec bash -c {recipe.sh $(klist | awk -F'[ :@]+' '/Default principal/{print $3}') >/dev/null}

source $autorunConf
if [file exists $autorunConfPrivate] { source $autorunConfPrivate }

if ![info exist maxAbortRetry] {
	set maxAbortRetry 6
}
foreach {key val} [array get HostAvilable] { set HostMax($key) [expr int(ceil(($val+5)/10))] }

#parray HostMax
array set HostUsed {}
set alone {}

proc getXML {jobID} {
	set xml [exec bkr job-results --prettyxml $jobID]
	regsub {<\?xml .*?\?>} $xml {}
}

proc limitSubmitJobs {distro rgset arch testlist limitNum} {
	foreach file [glob -nocomplain *.xml] { file delete -force $file }
	set testNum [llength $testlist]
	if {$testNum == 0 || $limitNum < 0} {
		puts "\[bkr-autorun-monitor\]: {$testNum/$limitNum}limitSubmit($arch) -> testlist nil or no host available($limitNum)"
		return 0
	}

	# call bkr-runtest -n, and submit xml
	set cnt 0
	global alone
	if {![catch {set fp [open "|bkr-runtest -n $distro - ${alone} $rgset --random" w]} err]} {
		foreach T $testlist { puts $fp $T }
		catch {close $fp} err
		foreach xml [glob -nocomplain *.xml] {
			if {$cnt >= $limitNum} break
			puts "\[bkr-autorun-monitor\]: {$cnt/$limitNum} limitSubmit($arch) -> submit '$xml'"
			exec sh -c "mykinit.sh 2>/dev/null; :"
			set rc [catch {exec bkr job-submit $xml >>&jobsubmit.log} msg]
			if {$rc == 1} {
				puts "{Warn} bkr job-submit fail:\n$msg"
				exec bash -c {cat jobsubmit.log >>jobsubmit.err.log}
				continue
			}
			incr cnt [exec bash -c "grep -c '</recipe>' $xml; :"]
		}
	} else {
		puts "{WARN} limitSubmitJobs open|-> $err"
	}
	return $cnt
}

proc updateJobTask {jobid distro_rgset} {
	set XML [getXML $jobid]
	set doc [dom parse $XML]
	set root [$doc documentElement]
	set paramNodeList [$root selectNodes {//param[@name="TestID"]}]
	set testIDList {}
	foreach paramNode $paramNodeList {
		lappend testIDList [$paramNode @value]
	}
	set testIDList [lsort -unique $testIDList]

	foreach testID $testIDList {
		set tstat   {running}
		set rstat   {}
		set tres    {}
		set taskuri {}
		set recipeIdx 0
		foreach paramNode [$root selectNodes {//param[@value=$testID]}] {
			set taskNode [[$paramNode parentNode] parentNode]
			set recipeNode [$taskNode parentNode]
			set recipeid [$recipeNode @id]
			set kcovendNodes [$recipeNode selectNodes {//task[@name="/kernel/kcov/end"]}]

			if {$kcovendNodes != {}} {
				set endNode [lindex $kcovendNodes $recipeIdx]
				set res [$endNode @result]
				set stat [$endNode @status]
			} else {
				set res [$taskNode @result]
				set stat [$taskNode @status]
			}
			lappend tres "$res\([string range $stat 0 2])"
			lappend rstat $stat
			lappend taskuri "${recipeid}#task[$taskNode @id]"
			incr recipeIdx
		}
		set updateTestrun {
			update testrun
			set jobid=$jobid, testStat=$tstat, res=$tres, rstat=$rstat, taskuri=$taskuri
			where testid = $testID and distro_rgset = $distro_rgset
		}
		db eval $updateTestrun
	}
}

#_main_
set USER $tcl_platform(user)
set rc [catch {exec ps -U $USER -u $USER -o pid,user:20,cmd | grep -v grep | grep {tclsh.*bkr-autorun-monitor}} msg]
if {$rc == 0 && [pid] != [lindex $msg 0]} {
	puts "{Warn} bkr-autorun-monitor is running! $msg"
	exit 0
}
cd [dbroot]
sqlite3 db testrun.db
db timeout 6000

#Get running test list
array set runingTest {}
db eval {
    select
	trun.jobid as jobid,
	trun.testid as tid,
	trun.distro_rgset as distro_rgset,
	trun.abortedCnt as abortedCnt
    from testrun trun
    where trun.testStat = 'running' or trun.res LIKE '%New%'
} {
	lappend runningTest($jobid) [list $tid $distro_rgset $abortedCnt]
}

#Update test status and result
foreach {jobID taskList} [array get runningTest] {
	#ifdebug
	puts "==>$jobID"
	set XML [getXML $jobID]
	set doc [dom parse $XML]
	set root [$doc documentElement]
	set recipeNodeList [$root selectNodes {//recipe}]
	set recipeNum [llength $recipeNodeList]
	set completeStr [lrepeat $recipeNum Completed]
	foreach recipeNode $recipeNodeList {
		set arch [$recipeNode @arch]
		set _arch [lindex [split $arch .] 0]
		incr HostUsed($_arch)
		#puts "$jobID incr HostUsed($_arch) = $HostUsed($_arch)"
	}

	foreach taskinfo $taskList {
		lassign $taskinfo testID distro_rgset abortedCnt
		set tstat   {running}
		set rstat   {}
		set tres    {}
		set tresdetail {}
		set recipeIdx 0
		foreach paramNode [$root selectNodes {//param[@value=$testID]}] {
			set taskNode [[$paramNode parentNode] parentNode]
			#ifdebug
			puts [$taskNode asXML]
			set name [$taskNode @name]
			set recipeNode [$taskNode parentNode]

			set kcovendNodes [$recipeNode selectNodes {//task[@name="/kernel/kcov/end"]}]
			if {$kcovendNodes != {}} {
				set taskNode [lindex $kcovendNodes $recipeIdx]
			}

			set res  [$taskNode @result]
			set stat [$taskNode @status]

			set recipeid [$recipeNode @id]
			set taskid [$taskNode @id]
			set role [$taskNode @role]
			set arch [$recipeNode @arch]

			# process res
			set resdetailHead "${res}:"
			set resdetailTail ""
			if {$res != "Pass"} {
				if {$res == "Fail"} {
					set resdetailHead "Pass-:"
				}
				set taskResults [$taskNode selectNodes results]
				if {$taskResults != ""} {
					foreach taskResNode [$taskResults childNodes] {
						if [$taskResNode hasAttribute result] {
							set _res [$taskResNode @result]
						} else {
							set _res [string map {( {} ) {}} [$taskResNode text]]
						}
						set _path [$taskResNode @path]
						if ![string match */avc $_path] {
							if {$_res == "Fail"} {
								set resdetailHead "Fail:"
							} elseif {$_res == "Warn" && $resdetailHead == "Pass-:"} {
								set resdetailHead "Warn:"
							}
						}
						set path "  $_path $_res"
						append resdetailTail "$path\n"
					}
				}
			}

			#if just avc fail, set res pass
			if {[string match Pass* $resdetailHead] && $res == "Fail"} {
				set res "Pass"
			}

			append tresdetail "$resdetailHead\n$resdetailTail"

			set tasklog [exec bkr job-logs T:$taskid]
			if {$stat == "Aborted" && $tasklog != ""} {
				set stat "Completed"
			}

			# process stat
			if {$stat != "Completed"} {
				set res "$res\([string range $stat 0 2])"
			}
			lappend rstat $stat
			lappend tres  $res

			# if not pass get log
			if {$stat == "Completed"} {
				if {$res != "Pass"} {
					set logpath "log/[string map {{ } _} $distro_rgset]/$name-$arch-$testID/$jobID-$role-$recipeid-$res"
					#set rc [catch {exec bash -c "mkdir -p $logpath; cd $logpath; wget -N $tasklog; :"} msg]
				}
				# fix me wget special log file, and save to treddetail
			}
			incr recipeIdx
		}
		if {$rstat == $completeStr && [regexp {New} $tres] != 1} {
			set tstat completed
			set abortedCnt 0
		}
		if {"Aborted" in [concat $rstat]} {
			set tstat aborted
			incr abortedCnt 

			# cancel RecipeSet if task aborted
			set recipeSetNode [$recipeNode parentNode]
			set rsid [$recipeSetNode @id]
			exec bash -c "bkr job-cancel RS:$rsid &"
		}
		if {"Cancelled" in [concat $rstat]} {
			set tstat cancelled
		}
		set updateTestrun {
			update testrun
			set testStat=$tstat, res=$tres, rstat=$rstat, abortedCnt=$abortedCnt, resdetail=$tresdetail
			where testid = $testID and distro_rgset = $distro_rgset
		}
		db eval $updateTestrun
	}
}

#Get the test list that not submitted and aborted
set pkg [lindex $::argv 0]
if {$pkg == ""} {set pkg {%}}
set run [lindex $::argv 1]
if {$run == ""} {set run {%}}
parray HostUsed
puts "{Debug}=> pkg=$pkg; run=$run"
array set waitSubmitedTest {}
db eval {
    select
	trun.distro_rgset as distro_rgset, ti.test as test, trun.testid as testid, trun.abortedCnt as abrtCnt
    from testrun trun
    join testinfo ti on
        ti.pkgName LIKE $pkg and trun.distro_rgset LIKE $run and
        trun.testid = ti.testid and (trun.testStat = '' or trun.testStat = 'aborted' or trun.testStat = 'cancelled' or trun.jobid = '') and trun.abortedCnt < $maxAbortRetry
} {
puts "{Debug}=> $test; abrtCnt = $abrtCnt /$maxAbortRetry"
	set alone {}
	if {$abrtCnt > $maxAbortRetry} {continue}
	if {$abrtCnt > $maxAbortRetry/2} {set alone {-alone}}
	if ![regexp -- {-arch=([a-z0-9_]+)} $test _arch arch] {
		set arch x86_64
	}
	if {$arch == "auto"} {set arch x86_64}
	set distro [lindex $distro_rgset 0]
	set rgset  [lrange $distro_rgset 1 end]

	if [regexp -- {-arch=([a-z0-9_]+)} $rgset] {
		regexp -- {-arch=([a-z0-9_]+)} $rgset _arch arch
	}

	set distro_arch_rgset "$distro $arch $rgset"

	set tdict [lindex [::yaml::yaml2dict $test] 1]
	if [dict exist $tdict param] {
		regsub -line {param: \[} $test "&TestID=$testid, " ntest
	} else {
		regsub -line "\}$"  $test  ", param: \[TestID=$testid]&" ntest
	}
	lappend waitSubmitedTest($distro_arch_rgset) $ntest
}

#Submit the test list to beaker: limitSubmitJobs
foreach {distro_arch_rgset testList} [array get waitSubmitedTest] {
	set distro [lindex $distro_arch_rgset 0]
	set arch [lindex $distro_arch_rgset 1]
	set rgset [lrange $distro_arch_rgset 2 end]
	if {$arch == "auto"} {set arch x86_64}

	puts "\n\[bkr-autorun-monitor\]: Submit {Distro arch rgset: ($distro) ($arch) ($rgset)}"

	close [open jobsubmit.log w+]

	set _arch [lindex [split $arch .] 0]
	if ![info exist HostUsed($_arch)] { set HostUsed($_arch) 0 }
	set limitNum [expr $HostMax($_arch) - $HostUsed($_arch)]

	set N [limitSubmitJobs $distro $rgset $arch $testList $limitNum]
	if {$N < 0} {set N 0}
	set HostUsed($_arch) [expr $HostUsed($_arch) + $N]

	set JobList [list {*}[exec bash -c {egrep -o J:[0-9]+ jobsubmit.log; :}]]
	set distro_rgset $distro
	if {$rgset != ""} { append distro_rgset " $rgset" }
	db transaction {
		foreach jobid $JobList {
			updateJobTask $jobid $distro_rgset
		}
	}
}

